import sys
import nltk.classify.util
import nltk.tokenize
import TextSanitizationHelpers
from nltk.classify import NaiveBayesClassifier
from nltk.corpus import movie_reviews
from nltk.corpus import stopwords

def TweetPreprocessing(tweet):
    tweet = TextSanitizationHelpers.ReplaceEmoticons(tweet)
    tweet = TextSanitizationHelpers.ReplaceUrl(tweet)    
    tweet = TextSanitizationHelpers.ReplacePointers(tweet)
    tweet = TextSanitizationHelpers.ReplaceRepeatedLetters(tweet)    
    tweet = TextSanitizationHelpers.ApplySteeming(tweet)    
    tweet = TextSanitizationHelpers.ReplacePunctuation(tweet)    
    tweet = tweet.lower()            
    tokenizedTweet = nltk.word_tokenize(tweet)         
    return word_feats(tokenizedTweet)    

def word_feats(words):
    stopset = set(stopwords.words('english')) 
    return dict([(word, True) for word in words if word not in stopset])

print
print '-- Preprocesando el set de entrenamiento --'
print 
positiveTweetsFeats = []
negativeTweetsFeats = []

for tweet in open("TweetsCorpus.neg.txt", 'r'):    
    tweet = tweet.replace("\n","")    
    tweetPreprocessed = TweetPreprocessing(tweet)                    
    negativeTweetsFeats.append((tweetPreprocessed,'negativo'))
 
for tweet in open("TweetsCorpus.pos.txt", 'r'):    
    tweet = tweet.replace("\n","")    
    tweetPreprocessed = TweetPreprocessing(tweet)                    
    positiveTweetsFeats.append((tweetPreprocessed,'positivo'))
    
negcutoff = len(negativeTweetsFeats)*3/4
poscutoff = len(positiveTweetsFeats)*3/4
 
trainfeats = negativeTweetsFeats[:negcutoff] + positiveTweetsFeats[:poscutoff]
testfeats = negativeTweetsFeats[negcutoff:] + positiveTweetsFeats[poscutoff:]

print
print '-- Entrenamos la red --'
print 'Utilizando %d mensajes de entrenamiento, y %d mensajes de prueba' % (len(trainfeats), len(testfeats))
print 

classifier = NaiveBayesClassifier.train(trainfeats)
print 'La accuracy es:', nltk.classify.util.accuracy(classifier, testfeats)
print
print 'Las palabras mas reelevantes son:'
print
classifier.show_most_informative_features()

for tweet in open("tweets.txt", 'r'):
    tweet = tweet.replace("\n","")    
    tweetPreprocessed = TweetPreprocessing(tweet)      
    value = classifier.classify(tweetPreprocessed)
    print           
    print 'La clasificacion del tweet "%s" es:' % tweet
    print value

raw_input('Presione una tecla para salir...')
